package com.moneycollect.webspider.google.shopify;

import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.fasterxml.jackson.core.type.TypeReference;
import com.moneycollect.webspider.atome.entity.ContactEntity;
import com.moneycollect.webspider.builtwith.entity.BuildWithExportEntity;
import com.moneycollect.webspider.builtwith.entity.SiteInfoEntity;
import com.moneycollect.webspider.common.*;
import com.moneycollect.webspider.google.entity.BingShopifyExportEntity;
import com.moneycollect.webspider.google.entity.WebSiteInfo;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.CollectionUtils;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * @author xuhuiqian
 * @since 2021/12/8 18:17
 */
public class GoogleWebSpider {
    private static final List<String> KEY_WORD_LIST = List.of("featured",
            "women",
            "beauty",
            "men",
            "baby",
            "kids",
            "home decor",
            "electronics",
            "food",
            "drinks",
            "travel",
            "sports",
            "outdoor",
            "health",
            "lifestyle",
            "fashion",
            "accessories",
            "in-store");

    public static void  run () {
       /* try {
            getSaasPlatformSites();
        } catch (Exception e) {
            e.printStackTrace();
        }*/

        /*try {
            fixWebSiteInfo();
        } catch (Exception e) {
            e.printStackTrace();
        }*/

       /* try {
            getContact();
        } catch (Exception e) {
            e.printStackTrace();
        }*/

        /*try {
            getContactContent();
        } catch (Exception e) {
            e.printStackTrace();
        }*/

        try {
            filterResult();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    // region 获取商家站点列表
    private static void getSaasPlatformSites() {
        List<WebSiteInfo> result = new ArrayList<>();
        for(var kw : KEY_WORD_LIST) {
            System.out.println("key word:" + kw);

            //var searchText = "inurl:myshopify.com " + kw + "  hong kong";
            var searchText = kw + " hong kong  site:myshopify.com";

            var command = new ArrayList<String>();
            // command.add("--headless");//不显示google 浏览器
            Launcher launcher = new Launcher();
            try (SessionFactory factory = launcher.launch(command);
                 Session session = factory.create()) {
                session.navigate("https://www.bing.com/");
                session.waitDocumentReady(60000);
                session.click("#sb_form_q")
                        .sendKeys(searchText)
                        .sendEnter().waitDocumentReady(60000);

                var isMatch = session.matches(".sb_pagN_bp");
                var matchCount = 1;
                getWebSiteFromPageHtmlContent(session.getContent(), result, kw);
                System.out.println("key word:" + kw + ", page:" + matchCount);

                while (isMatch && matchCount < 20){
                    session.click(".sb_pagN_bp").waitDocumentReady(60000);
                    getWebSiteFromPageHtmlContent(session.getContent(), result, kw);

                    matchCount++;
                    System.out.println("key word:" + kw + ", page:" + matchCount);

                    isMatch = session.matches(".sb_pagN_bp");
                    session.wait(1000);
                }
            } catch (Exception ex) {
                ex.printStackTrace();
            } finally {
                launcher.kill();
            }

        }

        var jsonResult = JsonUtil.toJson(result);
        try {
            FileHelper.saveFile(jsonResult, "E:\\bing_shopify_list.txt");
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private static void getWebSiteFromPageHtmlContent(String htmlContent, List<WebSiteInfo> result, String kw) {
        Document doc = Jsoup.parseBodyFragment(htmlContent);
        var citeElements = doc.getElementsByTag("cite");
        for (var el : citeElements) {
            var citeText = el.text();
            if (citeText.contains("myshopify.com")) {
                WebSiteInfo entity = new WebSiteInfo();
                entity.setWebSiteUrl(citeText);
                entity.setSaasPlatform("shopify");
                entity.setType(kw);
                result.add(entity);
            }
        }
    }

    // endregion

    // region 对商家站点列表进行修正去重
    private static void fixWebSiteInfo() throws Exception {
        var merchantListJson = FileHelper.readAllFile4Utf8("E:\\bing_shopify_list.txt");
        var brandList = JsonUtil.parseObject(merchantListJson, new TypeReference<List<WebSiteInfo>>() {
        });

        List<WebSiteInfo> newList = new ArrayList<>();
        assert brandList != null;
        for (var b : brandList) {
            var idx = b.getWebSiteUrl().lastIndexOf(".myshopify.com");
            var url = b.getWebSiteUrl().substring(0, idx + 14);

            var isExist = false;
            for(var e : newList) {
                if(e.getWebSiteUrl().equalsIgnoreCase(url)) {
                    isExist = true;
                    break;
                }
            }
            if(!isExist) {
                b.setWebSiteUrl(url);
                newList.add(b);
            }
        }

        var jsonResult = JsonUtil.toJson(newList);
        try {
            FileHelper.saveFile(jsonResult, "E:\\bing_shopify_list_fix.txt");
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    // endregion

    // region 获取商家联系页面的URL
    private static void getContact() throws Exception {
        var merchantListJson = FileHelper.readAllFile4Utf8("E:\\bing_shopify_list_fix.txt");
        var brandList = JsonUtil.parseObject(merchantListJson, new TypeReference<List<WebSiteInfo>>() {
        });

        if (!CollectionUtils.isEmpty(brandList)) {
            for (var b : brandList) {
                try {
                    if (StringUtils.isBlank(b.getWebSiteUrl()) || StringUtils.isNotBlank(b.getContactUsUrl())) {
                        continue;
                    }

                    String contactUsUrl = null;
                    var requestUrl = fixWebSiteUrl(b.getWebSiteUrl());

                    var htmlContent = GetHtmlHelper.getHomePageContentByHttpClient(requestUrl);
                    contactUsUrl = searchContactUsUrl(htmlContent);
                    if (StringUtils.isBlank(contactUsUrl)) {
                        htmlContent = GetHtmlHelper.getUrlHtmlContentByCdp(requestUrl);
                        contactUsUrl = searchContactUsUrl(htmlContent);
                    }

                    b.setContactUsUrl(contactUsUrl);
                    System.out.println("[" + b.getType() + "][" + b.getWebSiteUrl() + "]get contract url");
                } catch (Exception e) {
                    System.out.println("[" + b.getType() + "][" + b.getWebSiteUrl() + "]get contract url exception:" + e.getMessage());
                }
            }
        }

        var result = JsonUtil.toJson(brandList);
        FileHelper.saveFile(result, "E:\\bing_shopify_contact_us.txt");
    }

    private static final Pattern contactUsPattern = Pattern.compile("(聯繫我們|联系我们|聯絡我們|联络我们|Contact Us|联络|Contact)+");
    private static String searchContactUsUrl(String htmlDoc) {
        if (StringUtils.isBlank(htmlDoc)) {
            return null;
        }
        try {
            Document doc = Jsoup.parseBodyFragment(htmlDoc);
            var allElement = doc.getElementsMatchingText(contactUsPattern);
            if (allElement.size() > 0) {
                for (org.jsoup.nodes.Element element : allElement) {
                    var href = element.attr("href");
                    if (StringUtils.isNotBlank(href)) {
                        return href;
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }

    private static String fixWebSiteUrl(String url) {
        if(StringUtils.isBlank(url)) {
            return  url;
        }

        if(url.startsWith("http")) {
            return url;
        }
        return "https://" + url;
    }

    // endregion

    // region 从联系页面分析联系方式
    private static void getContactContent() throws Exception {
        var bankJson = FileHelper.readAllFile4Utf8("E:\\bing_shopify_contact_us.txt");
        var brandList = JsonUtil.parseObject(bankJson, new TypeReference<List<WebSiteInfo>>() {
        });

        if (!CollectionUtils.isEmpty(brandList)) {
            for (var b : brandList) {
                if (StringUtils.isBlank(b.getContactUsUrl())) {
                    continue;
                }

                var fixUrl = fixContactUsUrl(b.getWebSiteUrl(), b.getContactUsUrl());
                if (StringUtils.isBlank(fixUrl) || !fixUrl.toLowerCase().startsWith("http")) {
                    continue;
                }

                extractContactContent(b, fixUrl, null);
                System.out.println("[" + b.getSaasPlatform() + "][" + b.getWebSiteUrl() + "]get contract content");
            }
        }

        var result = JsonUtil.toJson(brandList);
        FileHelper.saveFile(result, "E:\\bing_shopify_contact.txt");
    }

    private static String fixContactUsUrl(String webSiteUrl, String contactUsUrl) {
        if (contactUsUrl.toLowerCase().startsWith("http")) {
            return contactUsUrl;
        }

        if (contactUsUrl.toLowerCase().startsWith("mailto")) {
            return null;
        }

        var root = StringUtils.stripEnd(webSiteUrl, "/");
        var relatedUrl = StringUtils.stripStart(contactUsUrl, "/");
        var resultUrl =  root + "/" + relatedUrl;
        if(!resultUrl.startsWith("http")) {
            resultUrl = "https://" + resultUrl;
        }

        return resultUrl;
    }

    private static void extractContactContent(WebSiteInfo b, String url, String content) {
        ContactEntity contactUsEntity = null;
        if (StringUtils.isNotBlank(content)) {
            contactUsEntity = HandlerContact.getContactContent(content);
        } else if(StringUtils.isNotBlank(url)) {
            String htmlContent = GetHtmlHelper.getHomePageContentByHttpClient(url);
            contactUsEntity = HandlerContact.getContactContent(htmlContent);
            if (CollectionUtils.isEmpty(contactUsEntity.getEmail()) && CollectionUtils.isEmpty(contactUsEntity.getPhone())) {
                htmlContent = GetHtmlHelper.getUrlHtmlContentByCdp(url);
                contactUsEntity = HandlerContact.getContactContent(htmlContent);
            }
        }

        if (b.getEmail() == null) {
            b.setEmail(new ArrayList<>());
        }
        if (b.getPhone() == null) {
            b.setPhone(new ArrayList<>());
        }

        if(contactUsEntity != null && contactUsEntity.getEmail() != null) {
            b.getEmail().addAll(contactUsEntity.getEmail());
        }

        if(contactUsEntity != null && contactUsEntity.getPhone() != null){
            b.getPhone().addAll(contactUsEntity.getPhone());
        }
    }

    // endregion

    // region 去重导出
    private static void filterResult() throws Exception {
        var bankJson = FileHelper.readAllFile4Utf8("E:\\bing_shopify_contact.txt");
        var brandList = JsonUtil.parseObject(bankJson, new TypeReference<List<WebSiteInfo>>() {
        });
        if (!CollectionUtils.isEmpty(brandList)) {
            for (var b : brandList) {
                HandlerContact.filterEmail(b);
                HandlerContact.filterPhone(b);
            }
        }

        var result = JsonUtil.toJson(brandList);
        FileHelper.saveFile(result, "E:\\bing_shopify_result.txt");

        exportData(brandList);
    }

    private static void exportData(List<WebSiteInfo> brandList) {
        String path = "E:\\bing_shopify.xls";
        ExcelWriter excelWriter = EasyExcel.write(path, BingShopifyExportEntity.class)
                .registerConverter(new EasyExcelListConvertor())
                .build();

        if (!CollectionUtils.isEmpty(brandList)) {
            writeSheet(brandList, excelWriter);
        }

        // 关闭
        excelWriter.finish();
        System.out.println("finish");
    }

    private static void writeSheet(List<WebSiteInfo> brandList, ExcelWriter excelWriter) {
        var cc = Comparator.<WebSiteInfo, Integer>comparing(p ->
        {
            if (CollectionUtils.isEmpty(p.getEmail()) && CollectionUtils.isEmpty(p.getPhone())) {
                return 1;
            }
            return 0;
        });

        //var sourceData = brandList.stream().sorted(cc).collect(Collectors.toList());
        WriteSheet sheet = EasyExcel.writerSheet("shopify").build();

        List<BingShopifyExportEntity> exportDataList = new ArrayList<>();
        for (var d : brandList) {
            var exportEntity = BingShopifyExportEntity.builder()
                    .webSiteUrl(d.getWebSiteUrl())
                    .type(d.getType())
                    .saasPlatform(d.getSaasPlatform())
                    .contactUsUrl(d.getContactUsUrl())
                    .email(d.getEmail())
                    .phone(d.getPhone())
                    .build();
            exportDataList.add(exportEntity);
        }

        excelWriter.write(exportDataList, sheet);

    }

    // endregion
}
